import scrapy
import requests
import re


class BaikeSpider(scrapy.Spider):
    name = "baike"
    allowed_domains = ["baike.biau.com"]
    start_urls = ["https://baike.baidu.com/item/%E5%91%A8%E6%9D%B0%E4%BC%A6/129156"]

    def parse(self, response):
        title = response.css('dl>dd>span>h1::text').get()
        print('####', title)

    def start_requests(self):
        for url in BaikeSpider.start_urls:
            access_url = url
            headers = {
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Edg/116.0.1938.62'
            }
            r = requests.get(access_url, headers=headers, allow_redirects=False)
            if r.status_code == 200:
                yield scrapy.Request(access_url, callback=self.extract_cnt, dont_filter=True)
            else:
                real_url = r.headers['Location']
                access_url = real_url
                # yield scrapy.Request(access_url, callback=self.extract_list, dont_filter=True, meta={'access_url': access_url})
                yield scrapy.Request(access_url, callback=self.parse, dont_filter=True)

    # 内容
    def extract_cnt(self, response):
        # item = Record()
        content_html = response.css('div.lemma-title-wrapper')
        title = content_html.xpath('./dl/dd/span/h1/text()').extract_first()
        print(title)
        tmp_data = content_html.xpath('.').extract_first()
        print('#### tmp_data', tmp_data)
        intro = re.findall(r'<div class="lemma-summary J-summary" label-module="lemmaSummary">(.*)<div class="lemmaWgt-promotion-leadPVBtn">',
                           tmp_data, re.S)
        sel = scrapy.Selector(text=intro[0])
        ptexts = sel.xpath('.').extract()
        intro = ''
        for ptext in ptexts:
            p_sel = scrapy.Selector(text=ptext)
            txts = p_sel.xpath('//text()').extract()
            txt_cnt = ''
            for txt in txts:
                txt_cnt = txt_cnt + txt
            txt_cnt = '    ' + txt_cnt + '\n'
            intro = intro + txt_cnt
            print(intro)
        cnt = re.findall('<div class="para MARK_MODULE" label-module="para" data-uuid="goesgliyeq" data-pid="7">(.*)</div>', tmp_data, re.S)
        sel = scrapy.Selector(text=cnt[0])
        ptexts = sel.xpath('//text()').extract()
        cnt = ''
        for ptext in ptexts:
            p_sel = scrapy.Selector(text=ptext)
            txts = p_sel.xpath('//text()').extract()
            txt_cnt = ''
            if len(txts) == 0:
                pass
            else:
                for txt in txts:
                    txt_cnt = txt_cnt + txt
                txt_cnt = '    ' + txt_cnt + '\n'
                cnt = cnt + txt_cnt
        print(cnt)


